package com.xiaotu.spider.baidu.pipeline;

import com.xiaotu.common.model.NewsModel;
import com.xiaotu.common.model.TVPubCommentModel;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DataRedisKeyUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.APipeline;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

@Component("BaiduPipeline")
public class BaiduPipeline extends APipeline
{

    @Override
    protected void doProcess(ResultItems resultItems, Task task)
    {
        List<NewsModel> newsList = resultItems.get(NewsModel.class.getName());
        if (newsList == null || newsList.isEmpty())
            return;
        Map<String, JSONObject> newsMap = newsList.stream().filter(news -> this.validateData(news)).collect(Collectors
                .toMap(news -> DataRedisKeyUtils.getBaiduNewsKey(news), news ->
                {
                    JSONObject target = JSONObject.fromObject(news);
                    target.put(NewsModel.NEWS_TV.equals(news.getType()) ? "tvId" : "starId", news.getId());
                    return target;
                }));
        Map<String, String> historyMap = newsMap.keySet().stream().collect(Collectors.toMap(mapKey -> mapKey, mapKey
                -> StringUtils.EMPTY));

        if (newsMap == null || newsMap.isEmpty())
            return;

        String key = DataRedisKeyUtils.getKey(Constants.DataRedisKey.Baidu_YYYYMMDD);

        cacheHandler.hMSet(key, newsMap);
        cacheHandler.hMSet(Constants.DataRedisKey.Baidu_History.toString(), historyMap);

        cacheHandler.setCacheTimeout(Constants.DataRedisKey.Baidu_History.toString());
        cacheHandler.setCacheTimeout(key);
    }

    /**
     * 校验新闻是否为有效数据
     *
     * @param news 新闻
     * @return 是否
     */
    private boolean validateData(NewsModel news)
    {
        if (!StringUtil.allNotEmpty(news.getType(), news.getId(), news.getUrl()))
            return false;
        String key = DataRedisKeyUtils.getBaiduNewsKey(news);// 获取唯一标识以判断是否已存在
        return !cacheHandler
                .hExists(Constants.DataRedisKey.Baidu_History.toString(), key);

    }
}
